{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FinBERT Example Notebook\n",
    "\n",
    "This notebooks shows how to train and use the FinBERT pre-trained language model for financial sentiment analysis."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modules "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:55:04.902740Z",
     "start_time": "2020-03-23T15:55:04.876252Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "import argparse\n",
    "import shutil\n",
    "import os\n",
    "import logging\n",
    "from textblob import TextBlob\n",
    "\n",
    "from pytorch_pretrained_bert.file_utils import PYTORCH_PRETRAINED_BERT_CACHE\n",
    "from pytorch_pretrained_bert.modeling import BertForSequenceClassification\n",
    "from pytorch_pretrained_bert.tokenization import BertTokenizer\n",
    "from pytorch_pretrained_bert.optimization import *\n",
    "\n",
    "from finbert.finbert import *\n",
    "import finbert.utils as tools\n",
    "from pprint import pprint\n",
    "from sklearn.metrics import classification_report\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "project_dir = Path.cwd().parent\n",
    "pd.set_option('max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:55:05.711210Z",
     "start_time": "2020-03-23T15:55:05.693609Z"
    }
   },
   "outputs": [],
   "source": [
    "logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',\n",
    "                    datefmt = '%m/%d/%Y %H:%M:%S',\n",
    "                    level = logging.ERROR)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setting path variables:\n",
    "1. `lm_path`: the path for the pre-trained language model (If vanilla Bert is used then no need to set this one).\n",
    "2. `cl_path`: the path where the classification model is saved.\n",
    "3. `cl_data_path`: the path of the directory that contains the data files of `train.csv`, `validation.csv`, `test.csv`.\n",
    "---\n",
    "\n",
    "In the initialization of `bertmodel`, we can either use the original pre-trained weights from Google by giving `bm = 'bert-base-uncased`, or our further pre-trained language model by `bm = lm_path`\n",
    "\n",
    "\n",
    "---\n",
    "All of the configurations with the model is controlled with the `config` variable. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:55:07.405597Z",
     "start_time": "2020-03-23T15:55:07.386378Z"
    }
   },
   "outputs": [],
   "source": [
    "lm_path = project_dir/'models'/'language_model'/'finbertTRC2'\n",
    "cl_path = project_dir/'models'/'classifier_model'/'finbert-sentiment'\n",
    "cl_data_path = project_dir/'data'/'sentiment_data'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Configuring training parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can find the explanations of the training parameters in the class docsctrings. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:55:12.378583Z",
     "start_time": "2020-03-23T15:55:09.196746Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:55:09 - INFO - pytorch_pretrained_bert.modeling -   loading archive file /home/ubuntu/public_final_test/github/models/language_model/finbertTRC2 from cache at /home/ubuntu/public_final_test/github/models/language_model/finbertTRC2\n",
      "03/23/2020 15:55:09 - INFO - pytorch_pretrained_bert.modeling -   Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "03/23/2020 15:55:12 - INFO - pytorch_pretrained_bert.modeling -   Weights of BertForSequenceClassification not initialized from pretrained model: ['classifier.weight', 'classifier.bias']\n",
      "03/23/2020 15:55:12 - INFO - pytorch_pretrained_bert.modeling -   Weights from pretrained model not used in BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n"
     ]
    }
   ],
   "source": [
    "# Clean the cl_path\n",
    "try:\n",
    "    shutil.rmtree(cl_path) \n",
    "except:\n",
    "    pass\n",
    "\n",
    "bertmodel = BertForSequenceClassification.from_pretrained(lm_path,cache_dir=None, num_labels=3)\n",
    "\n",
    "\n",
    "config = Config(   data_dir=cl_data_path,\n",
    "                   bert_model=bertmodel,\n",
    "                   num_train_epochs=4,\n",
    "                   model_dir=cl_path,\n",
    "                   max_seq_length = 48,\n",
    "                   train_batch_size = 32,\n",
    "                   learning_rate = 2e-5,\n",
    "                   output_mode='classification',\n",
    "                   warm_up_proportion=0.2,\n",
    "                   local_rank=-1,\n",
    "                   discriminate=True,\n",
    "                   gradual_unfreeze=True )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`finbert` is our main class that encapsulates all the functionality. The list of class labels should be given in the prepare_model method call with label_list parameter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:55:16.657078Z",
     "start_time": "2020-03-23T15:55:16.639644Z"
    }
   },
   "outputs": [],
   "source": [
    "finbert = FinBert(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:55:17.850734Z",
     "start_time": "2020-03-23T15:55:17.368073Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:55:17 - INFO - finbert.finbert -   device: cuda n_gpu: 1, distributed training: False, 16-bits training: False\n",
      "03/23/2020 15:55:17 - INFO - pytorch_pretrained_bert.tokenization -   loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /home/ubuntu/.pytorch_pretrained_bert/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n"
     ]
    }
   ],
   "source": [
    "finbert.prepare_model(label_list=['positive','negative','neutral'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fine-tune the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:55:19.395707Z",
     "start_time": "2020-03-23T15:55:19.349642Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Get the training examples\n",
    "train_data = finbert.get_data('train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:55:25.912424Z",
     "start_time": "2020-03-23T15:55:20.065887Z"
    }
   },
   "outputs": [],
   "source": [
    "model = finbert.create_the_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Optional] Fine-tune only a subset of the model\n",
    "The variable `freeze` determines the last layer (out of 12) to be freezed. You can skip this part if you want to fine-tune the whole model.\n",
    "\n",
    "<span style=\"color:red\">Important: </span>\n",
    "Execute this step if you want a shorter training time in the expense of accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is for fine-tuning a subset of the model.\n",
    "\n",
    "freeze = 11\n",
    "\n",
    "for param in model.bert.embeddings.parameters():\n",
    "    param.requires_grad = False\n",
    "    \n",
    "for i in range(freeze):\n",
    "    for param in model.bert.encoder.layer[i].parameters():\n",
    "        param.requires_grad = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:58:35.486890Z",
     "start_time": "2020-03-23T15:55:27.293772Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:55:27 - INFO - finbert.utils -   *** Example ***\n",
      "03/23/2020 15:55:27 - INFO - finbert.utils -   guid: train-1\n",
      "03/23/2020 15:55:27 - INFO - finbert.utils -   tokens: [CLS] weber convinced his friend ray os ##trom , who owned a lake street sporting goods store , to sell the lure ##s . [SEP]\n",
      "03/23/2020 15:55:27 - INFO - finbert.utils -   input_ids: 101 13351 6427 2010 2767 4097 9808 13887 1010 2040 3079 1037 2697 2395 7419 5350 3573 1010 2000 5271 1996 17256 2015 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:55:27 - INFO - finbert.utils -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:55:27 - INFO - finbert.utils -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:55:27 - INFO - finbert.utils -   label: neutral (id = 2)\n",
      "03/23/2020 15:55:28 - INFO - finbert.finbert -   ***** Loading data *****\n",
      "03/23/2020 15:55:28 - INFO - finbert.finbert -     Num examples = 3100\n",
      "03/23/2020 15:55:28 - INFO - finbert.finbert -     Batch size = 32\n",
      "03/23/2020 15:55:28 - INFO - finbert.finbert -     Num steps = 96\n",
      "Epoch:   0%|          | 0/4 [00:00<?, ?it/s]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ce2fd003cdd47e5b715fe33b82efa3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration', max=97, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:55:50 - INFO - finbert.utils -   *** Example ***\n",
      "03/23/2020 15:55:50 - INFO - finbert.utils -   guid: validation-1\n",
      "03/23/2020 15:55:50 - INFO - finbert.utils -   tokens: [CLS] the fair value of the company ' s investment properties grew to eu ##r 2 . 80 ##3 billion at the end of march 2009 from eu ##r 2 . 69 ##1 million a year ago . [SEP]\n",
      "03/23/2020 15:55:50 - INFO - finbert.utils -   input_ids: 101 1996 4189 3643 1997 1996 2194 1005 1055 5211 5144 3473 2000 7327 2099 1016 1012 3770 2509 4551 2012 1996 2203 1997 2233 2268 2013 7327 2099 1016 1012 6353 2487 2454 1037 2095 3283 1012 102 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:55:50 - INFO - finbert.utils -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:55:50 - INFO - finbert.utils -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:55:50 - INFO - finbert.utils -   label: positive (id = 0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:55:51 - INFO - finbert.finbert -   ***** Loading data *****\n",
      "03/23/2020 15:55:51 - INFO - finbert.finbert -     Num examples = 776\n",
      "03/23/2020 15:55:51 - INFO - finbert.finbert -     Batch size = 32\n",
      "03/23/2020 15:55:51 - INFO - finbert.finbert -     Num steps = 96\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0375203baca54b9facef788e1613276e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Validating', max=25, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Validation losses: [0.94504958152771]\n",
      "No best model found\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  25%|██▌       | 1/4 [00:27<01:23, 27.70s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbaab36671d34a17a94f78c3b01fdad6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration', max=97, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:56:31 - INFO - finbert.utils -   *** Example ***\n",
      "03/23/2020 15:56:31 - INFO - finbert.utils -   guid: validation-1\n",
      "03/23/2020 15:56:31 - INFO - finbert.utils -   tokens: [CLS] the fair value of the company ' s investment properties grew to eu ##r 2 . 80 ##3 billion at the end of march 2009 from eu ##r 2 . 69 ##1 million a year ago . [SEP]\n",
      "03/23/2020 15:56:31 - INFO - finbert.utils -   input_ids: 101 1996 4189 3643 1997 1996 2194 1005 1055 5211 5144 3473 2000 7327 2099 1016 1012 3770 2509 4551 2012 1996 2203 1997 2233 2268 2013 7327 2099 1016 1012 6353 2487 2454 1037 2095 3283 1012 102 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:56:31 - INFO - finbert.utils -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:56:31 - INFO - finbert.utils -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:56:31 - INFO - finbert.utils -   label: positive (id = 0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:56:32 - INFO - finbert.finbert -   ***** Loading data *****\n",
      "03/23/2020 15:56:32 - INFO - finbert.finbert -     Num examples = 776\n",
      "03/23/2020 15:56:32 - INFO - finbert.finbert -     Batch size = 32\n",
      "03/23/2020 15:56:32 - INFO - finbert.finbert -     Num steps = 96\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "021723d689e74be0aa1bedadd956ade8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Validating', max=25, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Validation losses: [0.94504958152771, 0.40575565814971926]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  50%|█████     | 2/4 [01:08<01:03, 31.72s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b6546975e0b4d008d49b38702c6ed0b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration', max=97, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:57:26 - INFO - finbert.utils -   *** Example ***\n",
      "03/23/2020 15:57:26 - INFO - finbert.utils -   guid: validation-1\n",
      "03/23/2020 15:57:26 - INFO - finbert.utils -   tokens: [CLS] the fair value of the company ' s investment properties grew to eu ##r 2 . 80 ##3 billion at the end of march 2009 from eu ##r 2 . 69 ##1 million a year ago . [SEP]\n",
      "03/23/2020 15:57:26 - INFO - finbert.utils -   input_ids: 101 1996 4189 3643 1997 1996 2194 1005 1055 5211 5144 3473 2000 7327 2099 1016 1012 3770 2509 4551 2012 1996 2203 1997 2233 2268 2013 7327 2099 1016 1012 6353 2487 2454 1037 2095 3283 1012 102 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:57:26 - INFO - finbert.utils -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:57:26 - INFO - finbert.utils -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:57:26 - INFO - finbert.utils -   label: positive (id = 0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:57:27 - INFO - finbert.finbert -   ***** Loading data *****\n",
      "03/23/2020 15:57:27 - INFO - finbert.finbert -     Num examples = 776\n",
      "03/23/2020 15:57:27 - INFO - finbert.finbert -     Batch size = 32\n",
      "03/23/2020 15:57:27 - INFO - finbert.finbert -     Num steps = 96\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c2d79bf86784acfa11cefe4987f3474",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Validating', max=25, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Validation losses: [0.94504958152771, 0.40575565814971926, 0.34499375462532045]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  75%|███████▌  | 3/4 [02:03<00:38, 38.71s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38803decdd57404b8478eaaf7bd854f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration', max=97, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:58:28 - WARNING - pytorch_pretrained_bert.optimization -   Training beyond specified 't_total'. Learning rate multiplier set to 0.0. Please set 't_total' of WarmupLinearSchedule correctly.\n",
      "03/23/2020 15:58:29 - WARNING - pytorch_pretrained_bert.optimization -   Training beyond specified 't_total'. Learning rate multiplier set to 0.0. Please set 't_total' of WarmupLinearSchedule correctly.\n",
      "03/23/2020 15:58:29 - WARNING - pytorch_pretrained_bert.optimization -   Training beyond specified 't_total'. Learning rate multiplier set to 0.0. Please set 't_total' of WarmupLinearSchedule correctly.\n",
      "03/23/2020 15:58:29 - INFO - finbert.utils -   *** Example ***\n",
      "03/23/2020 15:58:29 - INFO - finbert.utils -   guid: validation-1\n",
      "03/23/2020 15:58:29 - INFO - finbert.utils -   tokens: [CLS] the fair value of the company ' s investment properties grew to eu ##r 2 . 80 ##3 billion at the end of march 2009 from eu ##r 2 . 69 ##1 million a year ago . [SEP]\n",
      "03/23/2020 15:58:29 - INFO - finbert.utils -   input_ids: 101 1996 4189 3643 1997 1996 2194 1005 1055 5211 5144 3473 2000 7327 2099 1016 1012 3770 2509 4551 2012 1996 2203 1997 2233 2268 2013 7327 2099 1016 1012 6353 2487 2454 1037 2095 3283 1012 102 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:58:29 - INFO - finbert.utils -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:58:29 - INFO - finbert.utils -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:58:29 - INFO - finbert.utils -   label: positive (id = 0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:58:30 - INFO - finbert.finbert -   ***** Loading data *****\n",
      "03/23/2020 15:58:30 - INFO - finbert.finbert -     Num examples = 776\n",
      "03/23/2020 15:58:30 - INFO - finbert.finbert -     Batch size = 32\n",
      "03/23/2020 15:58:30 - INFO - finbert.finbert -     Num steps = 96\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c4fc50c52cb4bebbc436dc731b68443",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Validating', max=25, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch: 100%|██████████| 4/4 [03:06<00:00, 46.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Validation losses: [0.94504958152771, 0.40575565814971926, 0.34499375462532045, 0.34853786170482637]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "trained_model = finbert.train(train_examples = train_data, model = model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test the model\n",
    "\n",
    "`bert.evaluate` outputs the DataFrame, where true labels and logit values for each example is given"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:58:40.056789Z",
     "start_time": "2020-03-23T15:58:40.023198Z"
    }
   },
   "outputs": [],
   "source": [
    "test_data = finbert.get_data('test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:58:48.248044Z",
     "start_time": "2020-03-23T15:58:41.699009Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:58:41 - INFO - finbert.utils -   *** Example ***\n",
      "03/23/2020 15:58:41 - INFO - finbert.utils -   guid: test-1\n",
      "03/23/2020 15:58:41 - INFO - finbert.utils -   tokens: [CLS] alt ##imo and tel ##ias ##one ##ra said yesterday that us ##man ##ov would be welcome to join the new company . [SEP]\n",
      "03/23/2020 15:58:41 - INFO - finbert.utils -   input_ids: 101 12456 16339 1998 10093 7951 5643 2527 2056 7483 2008 2149 2386 4492 2052 2022 6160 2000 3693 1996 2047 2194 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:58:41 - INFO - finbert.utils -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:58:41 - INFO - finbert.utils -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:58:41 - INFO - finbert.utils -   label: positive (id = 0)\n",
      "03/23/2020 15:58:42 - INFO - finbert.finbert -   ***** Loading data *****\n",
      "03/23/2020 15:58:42 - INFO - finbert.finbert -     Num examples = 969\n",
      "03/23/2020 15:58:42 - INFO - finbert.finbert -     Batch size = 32\n",
      "03/23/2020 15:58:42 - INFO - finbert.finbert -     Num steps = 120\n",
      "03/23/2020 15:58:42 - INFO - finbert.finbert -   ***** Running evaluation ***** \n",
      "03/23/2020 15:58:42 - INFO - finbert.finbert -     Num examples = 969\n",
      "03/23/2020 15:58:42 - INFO - finbert.finbert -     Batch size = 32\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "572ea93df41949e7aae8ed6877fd63d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Testing', max=31, style=ProgressStyle(description_width='init…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "results = finbert.evaluate(examples=test_data, model=trained_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare the classification report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:58:51.361079Z",
     "start_time": "2020-03-23T15:58:51.339548Z"
    }
   },
   "outputs": [],
   "source": [
    "def report(df, cols=['label','prediction','logits']):\n",
    "    #print('Validation loss:{0:.2f}'.format(metrics['best_validation_loss']))\n",
    "    cs = CrossEntropyLoss(weight=finbert.class_weights)\n",
    "    loss = cs(torch.tensor(list(df[cols[2]])),torch.tensor(list(df[cols[0]])))\n",
    "    print(\"Loss:{0:.2f}\".format(loss))\n",
    "    print(\"Accuracy:{0:.2f}\".format((df[cols[0]] == df[cols[1]]).sum() / df.shape[0]) )\n",
    "    print(\"\\nClassification Report:\")\n",
    "    print(classification_report(df[cols[0]], df[cols[1]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:58:53.190447Z",
     "start_time": "2020-03-23T15:58:53.166729Z"
    }
   },
   "outputs": [],
   "source": [
    "results['prediction'] = results.predictions.apply(lambda x: np.argmax(x,axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:58:54.436270Z",
     "start_time": "2020-03-23T15:58:54.399174Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:0.36\n",
      "Accuracy:0.85\n",
      "\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.83      0.78       238\n",
      "           1       0.78      0.94      0.85       125\n",
      "           2       0.93      0.84      0.88       606\n",
      "\n",
      "    accuracy                           0.85       969\n",
      "   macro avg       0.81      0.87      0.84       969\n",
      "weighted avg       0.86      0.85      0.85       969\n",
      "\n"
     ]
    }
   ],
   "source": [
    "report(results,cols=['labels','prediction','predictions'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With the `predict` function, given a piece of text, we split it into a list of sentences and then predict sentiment for each sentence. The output is written into a dataframe. Predictions are represented in three different columns: \n",
    "\n",
    "1) `logit`: probabilities for each class\n",
    "\n",
    "2) `prediction`: predicted label\n",
    "\n",
    "3) `sentiment_score`: sentiment score calculated as: probability of positive - probability of negative\n",
    "\n",
    "Below we analyze a paragraph taken out of [this](https://www.economist.com/finance-and-economics/2019/01/03/a-profit-warning-from-apple-jolts-markets) article from The Economist. For comparison purposes, we also put the sentiments predicted with TextBlob.\n",
    "> Later that day Apple said it was revising down its earnings expectations in the fourth quarter of 2018, largely because of lower sales and signs of economic weakness in China. The news rapidly infected financial markets. Apple’s share price fell by around 7% in after-hours trading and the decline was extended to more than 10% when the market opened. The dollar fell by 3.7% against the yen in a matter of minutes after the announcement, before rapidly recovering some ground. Asian stockmarkets closed down on January 3rd and European ones opened lower. Yields on government bonds fell as investors fled to the traditional haven in a market storm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:59:03.875213Z",
     "start_time": "2020-03-23T15:59:03.857612Z"
    }
   },
   "outputs": [],
   "source": [
    "text = \"Later that day Apple said it was revising down its earnings expectations in \\\n",
    "the fourth quarter of 2018, largely because of lower sales and signs of economic weakness in China. \\\n",
    "The news rapidly infected financial markets. Apple’s share price fell by around 7% in after-hours \\\n",
    "trading and the decline was extended to more than 10% when the market opened. The dollar fell \\\n",
    "by 3.7% against the yen in a matter of minutes after the announcement, before rapidly recovering \\\n",
    "some ground. Asian stockmarkets closed down on January 3rd and European ones opened lower. \\\n",
    "Yields on government bonds fell as investors fled to the traditional haven in a market storm.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:59:16.246963Z",
     "start_time": "2020-03-23T15:59:13.285393Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:59:13 - INFO - pytorch_pretrained_bert.modeling -   loading archive file /home/ubuntu/public_final_test/github/models/classifier_model/finbert-sentiment from cache at /home/ubuntu/public_final_test/github/models/classifier_model/finbert-sentiment\n",
      "03/23/2020 15:59:13 - INFO - pytorch_pretrained_bert.modeling -   Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cl_path = project_dir/'models'/'classifier_model'/'finbert-sentiment'\n",
    "model = BertForSequenceClassification.from_pretrained(cl_path, cache_dir=None, num_labels=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:59:19.531663Z",
     "start_time": "2020-03-23T15:59:17.744984Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:59:18 - INFO - pytorch_pretrained_bert.tokenization -   loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /home/ubuntu/.pytorch_pretrained_bert/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
      "03/23/2020 15:59:18 - INFO - finbert.utils -   *** Example ***\n",
      "03/23/2020 15:59:18 - INFO - finbert.utils -   guid: 0\n",
      "03/23/2020 15:59:18 - INFO - finbert.utils -   tokens: [CLS] later that day apple said it was rev ##ising down its earnings expectations in the fourth quarter of 2018 , largely because of lower sales and signs of economic weakness in china . [SEP]\n",
      "03/23/2020 15:59:18 - INFO - finbert.utils -   input_ids: 101 2101 2008 2154 6207 2056 2009 2001 7065 9355 2091 2049 16565 10908 1999 1996 2959 4284 1997 2760 1010 4321 2138 1997 2896 4341 1998 5751 1997 3171 11251 1999 2859 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:59:18 - INFO - finbert.utils -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:59:18 - INFO - finbert.utils -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:59:18 - INFO - finbert.utils -   label: None (id = 9090)\n",
      "03/23/2020 15:59:19 - INFO - finbert.utils -   *** Example ***\n",
      "03/23/2020 15:59:19 - INFO - finbert.utils -   guid: 0\n",
      "03/23/2020 15:59:19 - INFO - finbert.utils -   tokens: [CLS] yields on government bonds fell as investors fled to the traditional haven in a market storm . [SEP]\n",
      "03/23/2020 15:59:19 - INFO - finbert.utils -   input_ids: 101 16189 2006 2231 9547 3062 2004 9387 6783 2000 1996 3151 4033 1999 1037 3006 4040 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:59:19 - INFO - finbert.utils -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:59:19 - INFO - finbert.utils -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:59:19 - INFO - finbert.utils -   label: None (id = 9090)\n"
     ]
    }
   ],
   "source": [
    "result = predict(text,model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:59:20.519047Z",
     "start_time": "2020-03-23T15:59:20.440450Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>logit</th>\n",
       "      <th>prediction</th>\n",
       "      <th>sentiment_score</th>\n",
       "      <th>textblob_prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Later that day Apple said it was revising down its earnings expectations in the fourth quarter of 2018, largely because of lower sales and signs of economic weakness in China.</td>\n",
       "      <td>[0.019688552, 0.943051, 0.03726054]</td>\n",
       "      <td>negative</td>\n",
       "      <td>-0.923362</td>\n",
       "      <td>0.051746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The news rapidly infected financial markets.</td>\n",
       "      <td>[0.03636261, 0.8453178, 0.11831961]</td>\n",
       "      <td>negative</td>\n",
       "      <td>-0.808955</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Apple’s share price fell by around 7% in after-hours trading and the decline was extended to more than 10% when the market opened.</td>\n",
       "      <td>[0.011015116, 0.97663385, 0.012351025]</td>\n",
       "      <td>negative</td>\n",
       "      <td>-0.965619</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The dollar fell by 3.7% against the yen in a matter of minutes after the announcement, before rapidly recovering some ground.</td>\n",
       "      <td>[0.041714344, 0.94278675, 0.0154988365]</td>\n",
       "      <td>negative</td>\n",
       "      <td>-0.901072</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Asian stockmarkets closed down on January 3rd and European ones opened lower.</td>\n",
       "      <td>[0.00964421, 0.9713175, 0.019038342]</td>\n",
       "      <td>negative</td>\n",
       "      <td>-0.961673</td>\n",
       "      <td>-0.051111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Yields on government bonds fell as investors fled to the traditional haven in a market storm.</td>\n",
       "      <td>[0.024553798, 0.93890834, 0.03653784]</td>\n",
       "      <td>negative</td>\n",
       "      <td>-0.914355</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                          sentence  \\\n",
       "0  Later that day Apple said it was revising down its earnings expectations in the fourth quarter of 2018, largely because of lower sales and signs of economic weakness in China.   \n",
       "1  The news rapidly infected financial markets.                                                                                                                                      \n",
       "2  Apple’s share price fell by around 7% in after-hours trading and the decline was extended to more than 10% when the market opened.                                                \n",
       "3  The dollar fell by 3.7% against the yen in a matter of minutes after the announcement, before rapidly recovering some ground.                                                     \n",
       "4  Asian stockmarkets closed down on January 3rd and European ones opened lower.                                                                                                     \n",
       "5  Yields on government bonds fell as investors fled to the traditional haven in a market storm.                                                                                     \n",
       "\n",
       "                                     logit prediction  sentiment_score  \\\n",
       "0  [0.019688552, 0.943051, 0.03726054]      negative  -0.923362          \n",
       "1  [0.03636261, 0.8453178, 0.11831961]      negative  -0.808955          \n",
       "2  [0.011015116, 0.97663385, 0.012351025]   negative  -0.965619          \n",
       "3  [0.041714344, 0.94278675, 0.0154988365]  negative  -0.901072          \n",
       "4  [0.00964421, 0.9713175, 0.019038342]     negative  -0.961673          \n",
       "5  [0.024553798, 0.93890834, 0.03653784]    negative  -0.914355          \n",
       "\n",
       "   textblob_prediction  \n",
       "0  0.051746             \n",
       "1  0.000000             \n",
       "2  0.500000             \n",
       "3  0.000000             \n",
       "4 -0.051111             \n",
       "5  0.000000             "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blob = TextBlob(text)\n",
    "result['textblob_prediction'] = [sentence.sentiment.polarity for sentence in blob.sentences]\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:59:38.737969Z",
     "start_time": "2020-03-23T15:59:38.718255Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average sentiment is -0.91.\n"
     ]
    }
   ],
   "source": [
    "print(f'Average sentiment is %.2f.' % (result.sentiment_score.mean()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here is another example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:59:45.922058Z",
     "start_time": "2020-03-23T15:59:45.904622Z"
    }
   },
   "outputs": [],
   "source": [
    "text2 = \"Shares in the spin-off of South African e-commerce group Naspers surged more than 25% \\\n",
    "in the first minutes of their market debut in Amsterdam on Wednesday. Bob van Dijk, CEO of \\\n",
    "Naspers and Prosus Group poses at Amsterdam's stock exchange, as Prosus begins trading on the \\\n",
    "Euronext stock exchange in Amsterdam, Netherlands, September 11, 2019. REUTERS/Piroschka van de Wouw \\\n",
    "Prosus comprises Naspers’ global empire of consumer internet assets, with the jewel in the crown a \\\n",
    "31% stake in Chinese tech titan Tencent. There is 'way more demand than is even available, so that’s \\\n",
    "good,' said the CEO of Euronext Amsterdam, Maurice van Tilburg. 'It’s going to be an interesting \\\n",
    "hour of trade after opening this morning.' Euronext had given an indicative price of 58.70 euros \\\n",
    "per share for Prosus, implying a market value of 95.3 billion euros ($105 billion). The shares \\\n",
    "jumped to 76 euros on opening and were trading at 75 euros at 0719 GMT.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:59:48.152474Z",
     "start_time": "2020-03-23T15:59:47.028417Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/23/2020 15:59:47 - INFO - pytorch_pretrained_bert.tokenization -   loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /home/ubuntu/.pytorch_pretrained_bert/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
      "03/23/2020 15:59:47 - INFO - finbert.utils -   *** Example ***\n",
      "03/23/2020 15:59:47 - INFO - finbert.utils -   guid: 0\n",
      "03/23/2020 15:59:47 - INFO - finbert.utils -   tokens: [CLS] shares in the spin - off of south african e - commerce group nas ##pers surged more than 25 % in the first minutes of their market debut in amsterdam on wednesday . [SEP]\n",
      "03/23/2020 15:59:47 - INFO - finbert.utils -   input_ids: 101 6661 1999 1996 6714 1011 2125 1997 2148 3060 1041 1011 6236 2177 17235 7347 18852 2062 2084 2423 1003 1999 1996 2034 2781 1997 2037 3006 2834 1999 7598 2006 9317 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:59:47 - INFO - finbert.utils -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:59:47 - INFO - finbert.utils -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:59:47 - INFO - finbert.utils -   label: None (id = 9090)\n",
      "03/23/2020 15:59:47 - INFO - finbert.utils -   *** Example ***\n",
      "03/23/2020 15:59:47 - INFO - finbert.utils -   guid: 0\n",
      "03/23/2020 15:59:47 - INFO - finbert.utils -   tokens: [CLS] euro ##ne ##xt had given an indicative price of 58 . 70 euros per share for pro ##sus , implying a market value of 95 . 3 billion euros ( $ 105 billion ) . [SEP]\n",
      "03/23/2020 15:59:47 - INFO - finbert.utils -   input_ids: 101 9944 2638 18413 2018 2445 2019 24668 3976 1997 5388 1012 3963 19329 2566 3745 2005 4013 13203 1010 20242 1037 3006 3643 1997 5345 1012 1017 4551 19329 1006 1002 8746 4551 1007 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:59:47 - INFO - finbert.utils -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:59:47 - INFO - finbert.utils -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/23/2020 15:59:47 - INFO - finbert.utils -   label: None (id = 9090)\n"
     ]
    }
   ],
   "source": [
    "result2 = predict(text2,model)\n",
    "blob = TextBlob(text2)\n",
    "result2['textblob_prediction'] = [sentence.sentiment.polarity for sentence in blob.sentences]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T15:59:50.428951Z",
     "start_time": "2020-03-23T15:59:50.402385Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>logit</th>\n",
       "      <th>prediction</th>\n",
       "      <th>sentiment_score</th>\n",
       "      <th>textblob_prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Shares in the spin-off of South African e-commerce group Naspers surged more than 25% in the first minutes of their market debut in Amsterdam on Wednesday.</td>\n",
       "      <td>[0.94092953, 0.020505333, 0.038565084]</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.920424</td>\n",
       "      <td>0.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Bob van Dijk, CEO of Naspers and Prosus Group poses at Amsterdam's stock exchange, as Prosus begins trading on the Euronext stock exchange in Amsterdam, Netherlands, September 11, 2019.</td>\n",
       "      <td>[0.038308024, 0.009095702, 0.9525962]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.029212</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>REUTERS/Piroschka van de Wouw Prosus comprises Naspers’ global empire of consumer internet assets, with the jewel in the crown a 31% stake in Chinese tech titan Tencent.</td>\n",
       "      <td>[0.081532344, 0.004831061, 0.91363657]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.076701</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>There is 'way more demand than is even available, so that’s good,' said the CEO of Euronext Amsterdam, Maurice van Tilburg.</td>\n",
       "      <td>[0.87810785, 0.012375466, 0.10951674]</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.865732</td>\n",
       "      <td>0.533333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>'It’s going to be an interesting hour of trade after opening this morning.'</td>\n",
       "      <td>[0.3788318, 0.020865666, 0.6003026]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.357966</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Euronext had given an indicative price of 58.70 euros per share for Prosus, implying a market value of 95.3 billion euros ($105 billion).</td>\n",
       "      <td>[0.13808341, 0.012395262, 0.84952134]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.125688</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>The shares jumped to 76 euros on opening and were trading at 75 euros at 0719 GMT.</td>\n",
       "      <td>[0.7504708, 0.018825786, 0.23070341]</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.731645</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                    sentence  \\\n",
       "0  Shares in the spin-off of South African e-commerce group Naspers surged more than 25% in the first minutes of their market debut in Amsterdam on Wednesday.                                 \n",
       "1  Bob van Dijk, CEO of Naspers and Prosus Group poses at Amsterdam's stock exchange, as Prosus begins trading on the Euronext stock exchange in Amsterdam, Netherlands, September 11, 2019.   \n",
       "2  REUTERS/Piroschka van de Wouw Prosus comprises Naspers’ global empire of consumer internet assets, with the jewel in the crown a 31% stake in Chinese tech titan Tencent.                   \n",
       "3  There is 'way more demand than is even available, so that’s good,' said the CEO of Euronext Amsterdam, Maurice van Tilburg.                                                                 \n",
       "4  'It’s going to be an interesting hour of trade after opening this morning.'                                                                                                                 \n",
       "5  Euronext had given an indicative price of 58.70 euros per share for Prosus, implying a market value of 95.3 billion euros ($105 billion).                                                   \n",
       "6  The shares jumped to 76 euros on opening and were trading at 75 euros at 0719 GMT.                                                                                                          \n",
       "\n",
       "                                    logit prediction  sentiment_score  \\\n",
       "0  [0.94092953, 0.020505333, 0.038565084]  positive   0.920424          \n",
       "1  [0.038308024, 0.009095702, 0.9525962]   neutral    0.029212          \n",
       "2  [0.081532344, 0.004831061, 0.91363657]  neutral    0.076701          \n",
       "3  [0.87810785, 0.012375466, 0.10951674]   positive   0.865732          \n",
       "4  [0.3788318, 0.020865666, 0.6003026]     neutral    0.357966          \n",
       "5  [0.13808341, 0.012395262, 0.84952134]   neutral    0.125688          \n",
       "6  [0.7504708, 0.018825786, 0.23070341]    positive   0.731645          \n",
       "\n",
       "   textblob_prediction  \n",
       "0  0.250000             \n",
       "1  0.000000             \n",
       "2  0.000000             \n",
       "3  0.533333             \n",
       "4  0.500000             \n",
       "5  0.000000             \n",
       "6  0.000000             "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-23T16:00:27.031491Z",
     "start_time": "2020-03-23T16:00:27.012639Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average sentiment is 0.44.\n"
     ]
    }
   ],
   "source": [
    "print(f'Average sentiment is %.2f.' % (result2.sentiment_score.mean()))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:finbert]",
   "language": "python",
   "name": "conda-env-finbert-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
